# -*- coding: utf-8 -*-
import os
import json
import multiprocessing


def open_txt(file_name):
    with open(file_name, 'r') as f:
        try:
            line = f.readline()
            while line:
                yield line.strip()
                line = f.readline()
        except:
            print('No value')


def conver_label(gt):
    cn = ['！', '（', '）', '：', '，', '．', '［', '］', '；', '％', '０', '１', '２', '３', '４', '５', '６', '７', '８', '９', '％', '－',
          'Ａ', 'Ｂ', 'Ｃ', 'Ｄ',
          'Ｅ',
          'Ｆ', 'Ｇ', 'Ｈ', 'Ｉ', 'Ｊ', 'Ｋ', 'Ｌ', 'Ｍ', 'Ｎ', 'Ｏ', 'Ｐ', 'Ｑ', 'Ｒ', 'Ｓ', 'Ｔ', 'Ｕ', 'Ｖ', 'Ｗ', 'Ｘ', 'Ｙ', 'Ｚ', 'ａ',
          'ｂ',
          'ｃ', 'ｄ', 'ｅ', 'ｆ', 'ｇ', 'ｈ', 'ｉ', 'ｊ', 'ｋ', 'ｌ', 'ｍ', 'ｎ', 'ｏ', 'ｐ', 'ｑ', 'ｒ', 'ｓ', 'ｔ', 'ｕ', 'ｖ', 'ｗ', 'ｗ',
          'ｙ',
          'ｚ']
    en = ['!', '(', ')', ':', ',', '.', '[', ']', ';', '%', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '%', '-',
          'A', 'B', 'C', 'D',
          'E',
          'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a',
          'b',
          'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x',
          'y',
          'z']
    gt = ''.join(gt.split()).strip()
    gt_tmp = list(gt)
    for i in range(len(gt_tmp)):
        for c in range(len(cn)):
            if gt_tmp[i] == cn[c]:
                gt_tmp[i] = en[c]
    gt_tmp = ''.join(gt_tmp)
    '''
    for i in range(len(gt_tmp)):
        if c in cn:
            j = cn.index(c)
            gt_tmp[i] = en[j]
    '''
    return gt_tmp


def show_files(path, all_files):
    # 首先遍历当前目录所有文件及文件夹
    file_list = os.listdir(path)
    # 准备循环判断每个元素是否是文件夹还是文件，是文件的话，把名称传入list，是文件夹的话，递归
    for file in file_list:
        # 利用os.path.join()方法取得路径全名，并存入cur_path变量，否则每次只能遍历一层目录
        cur_path = os.path.join(path, file)
        # 判断是否是文件夹
        if os.path.isdir(cur_path):
            show_files(cur_path, all_files)
        else:
            all_files.append(cur_path)
    return all_files


contents = show_files("/data1/lhw/wiki_zh/", [])


def get_lines(i):
    path = contents[i].strip()
    print(path)
    name1 = path.split('/')[-2]
    name2 = path.split('/')[-1]
    out_file_name = name1 + '_' + name2 + '.txt'
    out_file = open(os.path.join('./wiki_outlines', out_file_name), 'w')
    for line in open_txt(path):
        json_str = json.loads(line)
        out = str(json_str['text']).strip()
        out = conver_label(out)
        out = out.split(',')
        for out_1 in out:
            out_1 = out_1.split('。')
            for out_2 in out_1:
                out_2 = out_2.split('、')
                for c in out_2:
                    if 2 < len(c) < 30:
                        out_file.writelines(c.strip() + '\n')


pool = multiprocessing.Pool(processes=30)
results = []
for i in range(len(contents)):
    pool.apply_async(get_lines, (i,))
pool.close()
pool.join()
